# fake ftssearch module
# Copyright (C) 2010  Chris Clark

# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.

# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.

# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import os
import re
import shutil
import glob
from datetime import datetime
from cgi import escape as escape_html

# Tested with version 2.2.2 from hg 1025:10b3e7689475 (2011-08-29) http://bitbucket.org/mchaput/whoosh
import whoosh

import whoosh.index
import whoosh.qparser
import whoosh.highlight
from whoosh.fields import Schema, STORED, ID, KEYWORD, TEXT, NUMERIC

## used to monkey patch Whoosh
from heapq import nlargest
from whoosh.reading import IndexReader


def setup(dir_name):

    def safe_mkdir(newdir):
        result_dir = os.path.abspath(newdir)
        try:
            os.makedirs(result_dir)
        except OSError, info:
            if info.errno == 17 and os.path.isdir(result_dir):
                pass
            else:
                raise
                
    def index_content(index_name, content_path, note_encoding=None, filesystem_encoding=None, progess_callback=None, clear_existing_index=False):
        """Create whoosh index based on *.txt files in 'content_path'
        """
        note_encoding = note_encoding or 'latin1'  # nasty default
        filesystem_encoding = filesystem_encoding or note_encoding
        
        ## TODO handle update index
        schema = Schema(title=TEXT(stored=True),
                        content=TEXT(stored=True),
                        path=ID(stored=True),
                        indexed_date=TEXT(stored=True),  # TODO maybe make this a DATETIME?
                        file_modified_date=TEXT(stored=True),  # TODO maybe make this a DATETIME?
                        file_size=NUMERIC(stored=True),
                        )
        # TODO index_name may need some work, e.g. use that as a dir and create things inside it. maybe too complex though..
        #print 'versionstring', whoosh.versionstring()
        if not whoosh.index.exists_in(index_name) or clear_existing_index:
            safe_mkdir(index_name)
            # Create using a Schema object
            ix = whoosh.index.create_in(index_name, schema)
            
            # Add documents
            writer = ix.writer()
            
            file_count = 0
            for content_filename in glob.glob(os.path.join(content_path, '*.txt')):
                #print repr(content_filename) ## REMOVE BEFORE SUBMIT
                if filesystem_encoding:
                    content_filename = unicode(content_filename, filesystem_encoding)
                else:
                    content_filename = unicode(content_filename)
                if progess_callback:
                    progess_callback(filename=content_filename)

                tmp_text = open(content_filename, 'rb').read()
                if note_encoding:
                    tmp_text = unicode(tmp_text, note_encoding)
                else:
                    tmp_text = unicode(tmp_text)
                title = os.path.basename(content_filename)  # TODO more tricks...
                TIME_FORMAT_STR = '%Y-%m-%d %H:%M:%S'  # Vaguely ANSI like
                tmp_dict = dict(
                                title=title,
                                content=tmp_text,
                                path=content_filename,
                                indexed_date=unicode(datetime.now().strftime(TIME_FORMAT_STR)),  # TODO maybe make this a date?
                                file_modified_date=unicode(datetime.fromtimestamp(os.path.getmtime(content_filename)).strftime(TIME_FORMAT_STR)),  # TODO maybe make this a date?
                                file_size=os.path.getsize(content_filename),
                                )
                writer.add_document(**tmp_dict)
                file_count += 1
            writer.commit()  # TODO commit after each or defer, which is faster?
        else:
            raise NotImplemented()
            ix = whoosh.index.open_dir(index_name)
        ## TODO close / unlock?
        return file_count
    
    idx_name = dir_name + '_whooshidx'
    num_docs_indexed = index_content(idx_name, dir_name, clear_existing_index=True)
    fts_handle = idx_name
    return fts_handle


def search(fts_handle, search_term):
    """Returns html search results.
    Fake returns static results
    """
    
    # assert search_term is unicode type?
    index_name = fts_handle
    try:
        ix = whoosh.index.open_dir(index_name)
    except IOError:
        raise SearchException('IOError on full text search index open (missing or corrupt?)')

    ## iterate over schema and construct items to display (and order to display them)
    search_field_name = 'content'  # name of field to perform html highlighting on, assumed to be a stored field
    assert search_field_name in ix.schema.stored_names()
    
    schema_names_list = []
    for tmp_name in ix.schema.stored_names():
        if tmp_name != search_field_name:
            if isinstance(ix.schema[tmp_name], ID):
                schema_names_list.insert(0, tmp_name)
            else:
                schema_names_list.append(tmp_name)
        
    searcher = ix.searcher()

    # code below based on Whossh 2.2.2 docs on searching and "How to create highlighted search result excerpts"
    
    # Parse the user query 
    # See http://packages.python.org/Whoosh/parsing.html , e.g. MultifieldParser
    parser = whoosh.qparser.QueryParser(search_field_name, schema=ix.schema, termclass=whoosh.qparser.query.Variations)  # Variations means search for "frog" will match "frog" and "frogs" (and vice-versa)
    q = parser.parse(search_term)
    
    # Get the search results 
    r = searcher.search(q, limit=None)   # without limit=None, limited to 10 hits
    
    result = []
    result.append('<p>%d hits<p>' % len(r))
    
    """2.2.2 docs mention  extra controle over formatter but it doesn't seem to have any effect:
    # Allow larger fragments
    r.formatter.maxchars = 300  # this does not exist

    # Show more context before and after
    r.formatter.surround = 500  # nor this
    """

    for d in r:
        # The text argument to highlight is the stored text of the title 
        
        for tmp_name in schema_names_list:
            tmp_html = d[tmp_name]

            ## FIXME TODO escape tmp_html (convert to bytes?)
            result.append('%s: ' % escape_html(tmp_name))  # cgi escape?
            result.append(escape_html(unicode(tmp_html)))
            result.append('</br>')

        #tmp_html = d[search_field_name]  # raw content, NOTE this only works for STORED attributes
        tmp_html = d.highlights(search_field_name)  # html highlighted terms
        result.append(tmp_html)
        result.append('</br></br>')
    
    searcher.close()
    return ''.join(result)

    return '<p>Search not implemented, implement your own!</p>'

###########################################################


def most_frequent_terms_startswith(self, fieldid, startswith, number=5):
    """Yields the top 'number' most frequent terms that start with "startswith"
    in the given field as a series of (frequency, text) tuples.
    """
    return nlargest(number,
                    ((self.frequency("content", token), token)
                     for token
                     in self.expand_prefix("content", startswith)))
IndexReader.most_frequent_terms_startswith = most_frequent_terms_startswith  # Monkey patch Whoosh


def gen_find_top_search_terms(index_name, searchterm, maxnum=5):
    """TODO remove and use internal to Whoosh function, add in r164 in Whoosh svn. open search style finx pop search terms
    
    AIM of gen function is to create lookup class/function that already has index open
    for now this is not being done...... posibly performance improvement!
    """
    try:
        ix = whoosh.index.open_dir(index_name)
    except IOError:
        raise SearchException('IOError on full text search index open (missing or corrupt?)')

    searcher = ix.searcher()
    return searcher.ixreader.most_frequent_terms_startswith("content", searchterm, maxnum)
    #dummy_x = searcher.ixreader.most_frequent_terms_startswith("content", searchterm, maxnum)
    #print dummy_x
    #return dummy_x
